{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pima Indians Diabetes Data Set数据探索\n",
    "\n",
    "数据说明： Pima Indians Diabetes Data Set（皮马印第安人糖尿病数据集） 根据现有的医疗信息预测5年内皮马印第安人糖尿病发作的概率。\n",
    "\n",
    "数据集共9个字段: 0列为pregnants(怀孕次数)； 1列为Plasma_glucose_concentration(口服葡萄糖耐量试验中2小时后的血浆葡萄糖浓度)； 2列为blood_pressure(舒张压,单位:mm Hg） 3列为Triceps_skin_fold_thickness(三头肌皮褶厚度,单位：mm） 4列为serum_insulin(餐后血清胰岛素,单位:mm） 5列为BMI,体重指数（体重（公斤）/ 身高（米）^2） 6列为Diabetes_pedigree_function(糖尿病家系作用) 7列为Age(年龄) 8列为Target(分类变量,0或1）"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. import 工具包"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {},
   "outputs": [],
   "source": [
    "#首先 import 必要的模块\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "#color = sns.color_palette()\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. 读取数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pregnants</th>\n",
       "      <th>Plasma_glucose_concentration</th>\n",
       "      <th>blood_pressure</th>\n",
       "      <th>Triceps_skin_fold_thickness</th>\n",
       "      <th>serum_insulin</th>\n",
       "      <th>BMI</th>\n",
       "      <th>Diabetes_pedigree_function</th>\n",
       "      <th>Age</th>\n",
       "      <th>Target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.639947</td>\n",
       "      <td>0.866045</td>\n",
       "      <td>-0.031990</td>\n",
       "      <td>0.670643</td>\n",
       "      <td>-0.181541</td>\n",
       "      <td>0.166619</td>\n",
       "      <td>0.468492</td>\n",
       "      <td>1.425995</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.844885</td>\n",
       "      <td>-1.205066</td>\n",
       "      <td>-0.528319</td>\n",
       "      <td>-0.012301</td>\n",
       "      <td>-0.181541</td>\n",
       "      <td>-0.852200</td>\n",
       "      <td>-0.365061</td>\n",
       "      <td>-0.190672</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.233880</td>\n",
       "      <td>2.016662</td>\n",
       "      <td>-0.693761</td>\n",
       "      <td>-0.012301</td>\n",
       "      <td>-0.181541</td>\n",
       "      <td>-1.332500</td>\n",
       "      <td>0.604397</td>\n",
       "      <td>-0.105584</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.844885</td>\n",
       "      <td>-1.073567</td>\n",
       "      <td>-0.528319</td>\n",
       "      <td>-0.695245</td>\n",
       "      <td>-0.540642</td>\n",
       "      <td>-0.633881</td>\n",
       "      <td>-0.920763</td>\n",
       "      <td>-1.041549</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-1.141852</td>\n",
       "      <td>0.504422</td>\n",
       "      <td>-2.679076</td>\n",
       "      <td>0.670643</td>\n",
       "      <td>0.316566</td>\n",
       "      <td>1.549303</td>\n",
       "      <td>5.484909</td>\n",
       "      <td>-0.020496</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.342981</td>\n",
       "      <td>-0.185948</td>\n",
       "      <td>0.133453</td>\n",
       "      <td>-0.012301</td>\n",
       "      <td>-0.181541</td>\n",
       "      <td>-0.997745</td>\n",
       "      <td>-0.818079</td>\n",
       "      <td>-0.275760</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>-0.250952</td>\n",
       "      <td>-1.435189</td>\n",
       "      <td>-1.851862</td>\n",
       "      <td>0.329171</td>\n",
       "      <td>-0.610145</td>\n",
       "      <td>-0.211799</td>\n",
       "      <td>-0.676133</td>\n",
       "      <td>-0.616111</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1.827813</td>\n",
       "      <td>-0.218823</td>\n",
       "      <td>-0.031990</td>\n",
       "      <td>-0.012301</td>\n",
       "      <td>-0.181541</td>\n",
       "      <td>0.414047</td>\n",
       "      <td>-1.020427</td>\n",
       "      <td>-0.360847</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>-0.547919</td>\n",
       "      <td>2.476909</td>\n",
       "      <td>-0.197433</td>\n",
       "      <td>1.808882</td>\n",
       "      <td>4.660524</td>\n",
       "      <td>-0.284572</td>\n",
       "      <td>-0.947944</td>\n",
       "      <td>1.681259</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1.233880</td>\n",
       "      <td>0.109925</td>\n",
       "      <td>1.953325</td>\n",
       "      <td>-0.012301</td>\n",
       "      <td>-0.181541</td>\n",
       "      <td>-0.022590</td>\n",
       "      <td>-0.724455</td>\n",
       "      <td>1.766346</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   pregnants  Plasma_glucose_concentration  blood_pressure  \\\n",
       "0   0.639947                      0.866045       -0.031990   \n",
       "1  -0.844885                     -1.205066       -0.528319   \n",
       "2   1.233880                      2.016662       -0.693761   \n",
       "3  -0.844885                     -1.073567       -0.528319   \n",
       "4  -1.141852                      0.504422       -2.679076   \n",
       "5   0.342981                     -0.185948        0.133453   \n",
       "6  -0.250952                     -1.435189       -1.851862   \n",
       "7   1.827813                     -0.218823       -0.031990   \n",
       "8  -0.547919                      2.476909       -0.197433   \n",
       "9   1.233880                      0.109925        1.953325   \n",
       "\n",
       "   Triceps_skin_fold_thickness  serum_insulin       BMI  \\\n",
       "0                     0.670643      -0.181541  0.166619   \n",
       "1                    -0.012301      -0.181541 -0.852200   \n",
       "2                    -0.012301      -0.181541 -1.332500   \n",
       "3                    -0.695245      -0.540642 -0.633881   \n",
       "4                     0.670643       0.316566  1.549303   \n",
       "5                    -0.012301      -0.181541 -0.997745   \n",
       "6                     0.329171      -0.610145 -0.211799   \n",
       "7                    -0.012301      -0.181541  0.414047   \n",
       "8                     1.808882       4.660524 -0.284572   \n",
       "9                    -0.012301      -0.181541 -0.022590   \n",
       "\n",
       "   Diabetes_pedigree_function       Age  Target  \n",
       "0                    0.468492  1.425995       1  \n",
       "1                   -0.365061 -0.190672       0  \n",
       "2                    0.604397 -0.105584       1  \n",
       "3                   -0.920763 -1.041549       0  \n",
       "4                    5.484909 -0.020496       1  \n",
       "5                   -0.818079 -0.275760       0  \n",
       "6                   -0.676133 -0.616111       1  \n",
       "7                   -1.020427 -0.360847       0  \n",
       "8                   -0.947944  1.681259       1  \n",
       "9                   -0.724455  1.766346       1  "
      ]
     },
     "execution_count": 215,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#读取特征处理后的文件\n",
    "data = pd.read_csv(\"FE_pima-indians-diabetes.csv\")\n",
    "data.head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.分割数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_data = data['Target']\n",
    "X_data = data.drop(['Target'],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train: (614, 8)\n",
      "y_train: (614,)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/model_selection/_split.py:2069: FutureWarning: From version 0.21, test_size will always complement train_size unless both are specified.\n",
      "  FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "# 用train_test_split 分割训练数据和测试数据\n",
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_data, y_data, train_size = 0.8,random_state = 0)\n",
    "\n",
    "print(\"X_train:\",X_train.shape)\n",
    "print(\"y_train:\",y_train.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4.模型训练"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 默认参数的Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "lr = LogisticRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.53273727 0.42849684 0.48002428 0.55189027 0.46330807]\n",
      "cv logloss: 0.4912913453069324\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "#交叉验证用于评估模型性能、参数调优\n",
    "from sklearn.model_selection import cross_val_score\n",
    "loss = cross_val_score(lr,X_train,y_train,cv=5,scoring='neg_log_loss')\n",
    "\n",
    "print(-loss)\n",
    "print('cv logloss:',-loss.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7654723127035831\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "lr.fit(X_train,y_train)\n",
    "print(lr.score(X_train,y_train))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 正则化的 Logistic Regression及参数调优"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise-deprecating',\n",
       "       estimator=LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='warn',\n",
       "          n_jobs=None, penalty='l2', random_state=None, solver='warn',\n",
       "          tol=0.0001, verbose=0, warm_start=False),\n",
       "       fit_params=None, iid='warn', n_jobs=None,\n",
       "       param_grid={'penalty': ['l1', 'l2'], 'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 221,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "#带正则的参数调优\n",
    "penaltys = ['l1','l2']\n",
    "Cs = [0.001,0.01,0.1,1,10,100,1000]\n",
    "tuned_parameters = dict(penalty=penaltys,C=Cs)\n",
    "\n",
    "lr_penalty = LogisticRegression()\n",
    "grid = GridSearchCV(lr_penalty,tuned_parameters,cv=5,scoring='neg_log_loss')\n",
    "grid.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:125: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:125: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:125: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:125: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:125: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:125: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/yy/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:125: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([0.01051912, 0.00957375, 0.00969968, 0.00961118, 0.00901065,\n",
       "        0.00861564, 0.00958648, 0.0085217 , 0.00858788, 0.00898499,\n",
       "        0.00559602, 0.01110568, 0.01230464, 0.00998769]),\n",
       " 'std_fit_time': array([0.0043565 , 0.005951  , 0.00662242, 0.00259801, 0.00445765,\n",
       "        0.00362398, 0.00396878, 0.0038313 , 0.00513895, 0.00635661,\n",
       "        0.0028577 , 0.00531934, 0.00394618, 0.00224932]),\n",
       " 'mean_score_time': array([0.00148811, 0.00484481, 0.00633678, 0.00283527, 0.0058423 ,\n",
       "        0.00388174, 0.00475593, 0.00226974, 0.00547633, 0.01021714,\n",
       "        0.00328031, 0.00815778, 0.00355721, 0.00148778]),\n",
       " 'std_score_time': array([7.13359545e-05, 4.04125066e-03, 3.99486366e-03, 2.58337000e-03,\n",
       "        6.51485559e-03, 2.76978760e-03, 4.04870543e-03, 1.57530622e-03,\n",
       "        4.15859415e-03, 2.54214768e-03, 3.10032798e-03, 3.41487210e-03,\n",
       "        2.71813026e-03, 6.63555186e-05]),\n",
       " 'param_C': masked_array(data=[0.001, 0.001, 0.01, 0.01, 0.1, 0.1, 1, 1, 10, 10, 100,\n",
       "                    100, 1000, 1000],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_penalty': masked_array(data=['l1', 'l2', 'l1', 'l2', 'l1', 'l2', 'l1', 'l2', 'l1',\n",
       "                    'l2', 'l1', 'l2', 'l1', 'l2'],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'params': [{'C': 0.001, 'penalty': 'l1'},\n",
       "  {'C': 0.001, 'penalty': 'l2'},\n",
       "  {'C': 0.01, 'penalty': 'l1'},\n",
       "  {'C': 0.01, 'penalty': 'l2'},\n",
       "  {'C': 0.1, 'penalty': 'l1'},\n",
       "  {'C': 0.1, 'penalty': 'l2'},\n",
       "  {'C': 1, 'penalty': 'l1'},\n",
       "  {'C': 1, 'penalty': 'l2'},\n",
       "  {'C': 10, 'penalty': 'l1'},\n",
       "  {'C': 10, 'penalty': 'l2'},\n",
       "  {'C': 100, 'penalty': 'l1'},\n",
       "  {'C': 100, 'penalty': 'l2'},\n",
       "  {'C': 1000, 'penalty': 'l1'},\n",
       "  {'C': 1000, 'penalty': 'l2'}],\n",
       " 'split0_test_score': array([-0.69314718, -0.64303251, -0.6657806 , -0.54615594, -0.52195265,\n",
       "        -0.5241664 , -0.53165371, -0.53273727, -0.53437052, -0.53449463,\n",
       "        -0.53466711, -0.53468573, -0.53470351, -0.534705  ]),\n",
       " 'split1_test_score': array([-0.69314718, -0.63708374, -0.67208098, -0.5120819 , -0.45032059,\n",
       "        -0.44076291, -0.4291513 , -0.42849684, -0.42732761, -0.42724322,\n",
       "        -0.42712759, -0.42711848, -0.42710624, -0.42710601]),\n",
       " 'split2_test_score': array([-0.69314718, -0.64612165, -0.66526401, -0.53952508, -0.48887706,\n",
       "        -0.4837483 , -0.48056351, -0.48002428, -0.48014857, -0.48011158,\n",
       "        -0.48013317, -0.48012887, -0.48013198, -0.4801307 ]),\n",
       " 'split3_test_score': array([-0.69314718, -0.64647741, -0.66474318, -0.56184927, -0.54506156,\n",
       "        -0.54325404, -0.55205685, -0.55189027, -0.55372039, -0.55372713,\n",
       "        -0.55391313, -0.55392808, -0.5539425 , -0.55394836]),\n",
       " 'split4_test_score': array([-0.69314718, -0.63549412, -0.66974635, -0.51551355, -0.46935343,\n",
       "        -0.46672888, -0.46313495, -0.46330807, -0.4631867 , -0.46321848,\n",
       "        -0.46321201, -0.46321392, -0.46321327, -0.46321351]),\n",
       " 'mean_test_score': array([-0.69314718, -0.64164629, -0.66752109, -0.53503137, -0.49511738,\n",
       "        -0.49174174, -0.49132472, -0.49130573, -0.49176576, -0.49177417,\n",
       "        -0.49182584, -0.49183026, -0.49183476, -0.49183597]),\n",
       " 'std_test_score': array([0.        , 0.00455286, 0.00288354, 0.01879205, 0.03439134,\n",
       "        0.03738048, 0.04490878, 0.04524941, 0.04637226, 0.04641856,\n",
       "        0.04653185, 0.0465418 , 0.04655226, 0.04655419]),\n",
       " 'rank_test_score': array([14, 12, 13, 11, 10,  3,  2,  1,  4,  5,  6,  7,  8,  9],\n",
       "       dtype=int32),\n",
       " 'split0_train_score': array([-0.69314718, -0.63922272, -0.66263276, -0.52276722, -0.4748938 ,\n",
       "        -0.46729423, -0.46298223, -0.46280925, -0.46273166, -0.46272985,\n",
       "        -0.46272902, -0.46272899, -0.46272898, -0.46272898]),\n",
       " 'split1_train_score': array([-0.69314718, -0.64277716, -0.67644793, -0.53784265, -0.49883361,\n",
       "        -0.49252252, -0.48940391, -0.48926295, -0.48921017, -0.48920816,\n",
       "        -0.4892076 , -0.48920757, -0.48920756, -0.48920756]),\n",
       " 'split2_train_score': array([-0.69314718, -0.63851135, -0.66256296, -0.52993187, -0.48928845,\n",
       "        -0.48127195, -0.47757421, -0.47739535, -0.47732911, -0.4773273 ,\n",
       "        -0.47732657, -0.47732656, -0.47732655, -0.47732655]),\n",
       " 'split3_train_score': array([-0.69314718, -0.63732982, -0.65613244, -0.5188687 , -0.47023964,\n",
       "        -0.46438241, -0.46009427, -0.4599129 , -0.45983387, -0.45983205,\n",
       "        -0.45983118, -0.45983116, -0.45983115, -0.45983115]),\n",
       " 'split4_train_score': array([-0.69314718, -0.6433062 , -0.67346388, -0.53491287, -0.49331297,\n",
       "        -0.48548344, -0.48201337, -0.48188583, -0.48182708, -0.48182577,\n",
       "        -0.48182514, -0.48182512, -0.48182511, -0.48182511]),\n",
       " 'mean_train_score': array([-0.69314718, -0.64022945, -0.66624799, -0.52886466, -0.48531369,\n",
       "        -0.47819091, -0.4744136 , -0.47425325, -0.47418638, -0.47418462,\n",
       "        -0.4741839 , -0.47418388, -0.47418387, -0.47418387]),\n",
       " 'std_train_score': array([0.        , 0.00238035, 0.00755092, 0.00714734, 0.01093962,\n",
       "        0.01074692, 0.0112087 , 0.01122525, 0.01123545, 0.01123547,\n",
       "        0.01123558, 0.01123558, 0.01123558, 0.01123558])}"
      ]
     },
     "execution_count": 222,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best score： 0.49130572687725915\n",
      "best model: {'C': 1, 'penalty': 'l2'}\n"
     ]
    }
   ],
   "source": [
    "print('best score：',-grid.best_score_)\n",
    "print('best model:',grid.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.53273727 0.42849684 0.48002428 0.55189027 0.46330807]\n",
      "cv logloss: 0.4912913453069324\n"
     ]
    }
   ],
   "source": [
    "print(-loss)\n",
    "print('cv logloss:',-loss.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7654723127035831\n"
     ]
    }
   ],
   "source": [
    "print(lr.score(X_train,y_train))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "metadata": {},
   "outputs": [],
   "source": [
    "#from sklearn.metrics import accuracy_score\n",
    "\n",
    "#grid.best_estimator_\n",
    "#y_preds = grid.predict(X_test)\n",
    "\n",
    "#acc_score = accuracy_score(y_test,y_preds)\n",
    "#print(acc_score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LogisticRegressionCV及参数调优"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegressionCV(Cs=[0.001, 0.01, 0.1, 1, 10, 100, 1000],\n",
       "           class_weight=None, cv=5, dual=False, fit_intercept=True,\n",
       "           intercept_scaling=1.0, max_iter=100, multi_class='ovr',\n",
       "           n_jobs=None, penalty='l1', random_state=None, refit=True,\n",
       "           scoring='neg_log_loss', solver='liblinear', tol=0.0001,\n",
       "           verbose=0)"
      ]
     },
     "execution_count": 227,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegressionCV\n",
    "\n",
    "Cs = [0.001,0.01,0.1,1,10,100,1000]\n",
    "\n",
    "ls_L1 = LogisticRegressionCV(Cs=Cs,cv=5,penalty ='l1',solver='liblinear',scoring='neg_log_loss',multi_class='ovr')\n",
    "ls_L2 = LogisticRegressionCV(Cs=Cs,cv=5,penalty ='l2',solver='newton-cg',scoring='neg_log_loss',multi_class='ovr')\n",
    "\n",
    "ls_L1.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 228,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegressionCV(Cs=[0.001, 0.01, 0.1, 1, 10, 100, 1000],\n",
       "           class_weight=None, cv=5, dual=False, fit_intercept=True,\n",
       "           intercept_scaling=1.0, max_iter=100, multi_class='ovr',\n",
       "           n_jobs=None, penalty='l2', random_state=None, refit=True,\n",
       "           scoring='neg_log_loss', solver='newton-cg', tol=0.0001,\n",
       "           verbose=0)"
      ]
     },
     "execution_count": 228,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ls_L2.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{1: array([[-0.69314718, -0.6657806 , -0.52195205, -0.53165027, -0.5343715 ,\n",
       "         -0.53467043, -0.53470027],\n",
       "        [-0.69314718, -0.67208098, -0.45031448, -0.42914899, -0.4273248 ,\n",
       "         -0.42712811, -0.42710663],\n",
       "        [-0.69314718, -0.66526401, -0.48887631, -0.48056643, -0.48014495,\n",
       "         -0.48013246, -0.4801295 ],\n",
       "        [-0.69314718, -0.66474318, -0.5450606 , -0.55206138, -0.55372386,\n",
       "         -0.55391798, -0.55394505],\n",
       "        [-0.69314718, -0.66974635, -0.46934111, -0.4631371 , -0.46319337,\n",
       "         -0.46321636, -0.46321461]])}"
      ]
     },
     "execution_count": 229,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ls_L1.scores_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 237,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZMAAAELCAYAAAAcKWtPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3XucXWV97/HPd2653zMzQhJIAhMyIBBkCCgXCRMx9gKcqgjHKrQibZWCx5YKR49U1HPssVVrpa2ICNQLUDxIVGiEJCBSkExoSMh9CEISAhlyJeSe+Z0/1pqwM5lkZmfPzr7M9/167Vf2etZlfkvifLPWep71KCIwMzPLRUWhCzAzs9LnMDEzs5w5TMzMLGcOEzMzy5nDxMzMcuYwMTOznDlMzMwsZw4TMzPLmcPEzMxyVpXvHyBpBvCPQCVwR0R8rdP6bwLT0sWBQF1EDE/XXQV8IV33lYi4O20/E7gLGAA8DNwQ3QzlHz16dIwfP743TsnMrM+YP3/+GxFR2912yufrVCRVAiuA9wFrgHnAlRGx5BDb/yVwRkT8qaSRQAvQBAQwHzgzIjZJeha4HvgtSZh8OyIeOVwtTU1N0dLS0ktnZmbWN0iaHxFN3W2X79tcU4HWiFgVEbuBe4FLD7P9lcBP0u/vBx6NiI0RsQl4FJgh6RhgaEQ8k16N3ANclr9TMDOz7uQ7TMYAqzOW16RtB5F0PDABmNPNvmPS790e08zMjo5iegB/BfBAROzrrQNKulZSi6SWtra23jqsmZl1ku8wWQuMy1gem7Z15QrevsV1uH3Xpt+7PWZE3B4RTRHRVFvb7fMjMzM7QvkOk3lAg6QJkmpIAmNm540kTQZGAE9nNM8CLpY0QtII4GJgVkSsA7ZKOkeSgI8DD+X5PMzM7DDy2jU4IvZKuo4kGCqBOyNisaRbgZaI6AiWK4B7M7v3RsRGSV8mCSSAWyNiY/r9U7zdNfiR9GNmZgWS167BxcRdg83MslcsXYNL3i8WvkrL7zZ2v6GZWR+W9xHwpSwi+NZjK2ldv41zTxzF9Rc1cPbEUYUuy8ys6PjK5DAkMfO6c/nC7zey/LVtfOT2Z7ji9qd5+sUNhS7NzKyo+JlJD+3YvY+fPPsK//rEi6x/cxdTJ4zkhuYG3nPCKJJOZWZm5aenz0wcJlnauWcf981bzb88/iKvbd1J0/EjuGF6A+edONqhYmZlx2HSSW/35tq5Zx//3rKaf378RdZt2ckZxw3nhuYG3jup1qFiZmXDYdJJvroG79q7jwfmr+Gf577I2s07OH3ccG5oPpFpJ9U5VMys5DlMOsn3OJPde9v56XNruG1uK2s27eDUMcO4vrmB6Y0OFTMrXQ6TTo7WoMU9+9p58Lm1fGduK69s3M4pxw7l+uYG3tdYT0WFQ8XMSovDpJOjPQJ+z752HlrwKt+Zs5LfbdjO5HcM4YbmBt5/yjscKmZWMhwmnRTqdSp797Xz84Wv8k+zW1n1xlucVD+Ev2w+kd975zEOFTMreg6TTgr9bq597cEvFr7Kt2ev5MW2t2ioG8xfNjfw+6ceQ6VDxcyKlMOkk0KHSYd97cHDi9bxT3NWsuL1bZxQO4i/vKiBPzz9WIeKmRUdh0knxRImHdrbg0deeI1vz17J8tffZOLoQXx62olcOuVYqir9lhszKw4Ok06KLUw6tLcHv1ryGv84u5Wl67YyftRAPj3tRC47YwzVDhUzK7CieQW9pBmSlktqlXTTIba5XNISSYsl/ThtmyZpQcZnp6TL0nV3SXopY92UfJ9HvlRUiBnvPIaHrz+P2z92JoP6VXHjAwtp/ocnuG/eK+zZ117oEs3MupXXKxNJlcAK4H3AGpJZE6+MiCUZ2zQA9wMXRcQmSXURsb7TcUYCrcDYiNgu6S7gFxHxQE9rKdYrk84igjnL1vOPs1eycM0WxgwfwKennciHzhxLTZWvVMzs6CqWK5OpQGtErIqI3cC9wKWdtvkkcFtEbALoHCSpDwGPRMT2vFZbBCTR3FjPQ58+lx9cfRajh/Tjfz64iGl//zg/fOZldu3dV+gSzcwOku8wGQOszlhek7ZlmgRMkvSUpGckzejiOFcAP+nU9lVJCyV9U1K/3iu5OEhi2uQ6fvap93D3n06lfmg/vvCzF7jw649zz9O/Y+ceh4qZFY9iuG9SBTQAFwJXAt+TNLxjpaRjgFOBWRn73AxMBs4CRgKf6+rAkq6V1CKppa2tLT/V55kk3juplp/+xXv44SfOZuyIAXzxocW89+tz+cFTLzlUzKwo5DtM1gLjMpbHpm2Z1gAzI2JPRLxE8oylIWP95cCDEbGnoyEi1kViF/ADkttpB4mI2yOiKSKaamtre+F0CkcS5zWM5v4/ezc/vuZsjh81iC/9fAnn/9+5fP83L7Fjt0PFzAon32EyD2iQNEFSDcntqpmdtvkZyVUJkkaT3PZalbH+Sjrd4kqvVlDyOt7LgBfyUXwxksR7TkxC5d5rz+HE2sF8+RdJqHzv16vYvntvoUs0sz4or2ESEXuB60huUS0F7o+IxZJulXRJutksYIOkJcBc4MaI2AAgaTzJlc0TnQ79I0mLgEXAaOAr+TyPYnXOxFH85NpzuP/P3s3kdwzhqw8v5fy/m8u/PvEib+1yqJjZ0eNBi2Vk/ssb+dZjK3ly5RuMGFjNNedP5Kr3jGdwv6pCl2ZmJcoj4DvpC2HS4blXNvHt2St5fHkbwwdWc815E7jqPeMZ0r+60KWZWYlxmHTSl8Kkw/OrN/Pt2SuZvWw9Q/tX8YnzJvLnF06kX1VloUszsxLR0zDx/Y8ydvq44Xz/6rNYtGYL356zkm8+toLqKvGpC08sdGlmVmaKYZyJ5dmpY4fxvY83cfrYYTy65PVCl2NmZchh0odcNLmeBas388a2XYUuxczKjMOkD2lurCMC5izr6vVnZmZHzmHSh5xy7FCOGdaf2Ut9q8vMepfDpA+RxEWT63hy5Rt+p5eZ9SqHSR8zvbGe7bv38cyqDYUuxczKiMOkj3n3CaMYUF3J7KV+bmJmvcdh0sf0r67kvIbRzF76On1lwKqZ5Z/DpA+a3ljHq1t2snTdm4UuxczKhMOkD5o2uQ7AvbrMrNc4TPqguiH9OX3ccB7zeBMz6yUOkz5q+uQ6nl+9mbY3PRrezHLnMOmjLmpMbnXN9dWJmfWCvIeJpBmSlktqlXTTIba5XNISSYsl/TijfZ+kBelnZkb7BEm/TY95XzolsGXh5GOGcuyw/jzm5yZm1gvyGiaSKoHbgA8AJwNXSjq50zYNwM3AuRFxCvCZjNU7ImJK+rkko/3vgG9GxInAJuAT+TyPciSJixo9Gt7Meke+r0ymAq0RsSoidgP3Apd22uaTwG0RsQkgIg5730WSgIuAB9Kmu4HLerXqPqK5sZ4de/bxtEfDm1mO8h0mY4DVGctr0rZMk4BJkp6S9IykGRnr+ktqSds7AmMUsDki9h7mmNYD7544ioE1le4ibGY5K4aZFquABuBCYCzwa0mnRsRm4PiIWCtpIjBH0iJgS08PLOla4FqA4447rtcLL3X9qys578TRzFm6nrg0SC76zMyyl+8rk7XAuIzlsWlbpjXAzIjYExEvAStIwoWIWJv+uQp4HDgD2AAMl1R1mGOS7nd7RDRFRFNtbW3vnFGZmd5Yz6tbdrJk3dZCl2JmJSzfYTIPaEh7X9UAVwAzO23zM5KrEiSNJrnttUrSCEn9MtrPBZZE8kKpucCH0v2vAh7K83mUrWmT65Dwix/NLCd5DZP0ucZ1wCxgKXB/RCyWdKukjt5Zs4ANkpaQhMSNEbEBaARaJD2ftn8tIpak+3wO+KykVpJnKN/P53mUs9oh/Th97HA/NzGznKivvDm2qakpWlpaCl1GUfrOnJX8/a9W8Oznm6kb0r/Q5ZhZEZE0PyKautvOI+CN5sZ6wKPhzezIOUyMye8Yko6Gd5iY2ZFxmBiSaG6s5zceDW9mR8hhYgA0N9Ylo+Ff9Gh4M8uew8QAOCcdDe8XP5rZkXCYGJCMhj+/YTRzlq333PBmljWHie3X3FjPui07WfyqR8ObWXYcJrbfRR4Nb2ZHyGFi+40e3I8p44Yze5mfm5hZdhwmdoDpjfUsXLOF17fuLHQpZlZCHCZ2gGbPDW9mR8BhYgc4qX4IY4YP8Gh4M8uKw8QOkIyGr+M3rW0eDW9mPeYwsYM0N9azc087//niG4UuxcxKhMPEDnLOxJEMqqn0rS4z67G8h4mkGZKWS2qVdNMhtrlc0hJJiyX9OG2bIunptG2hpI9kbH+XpJckLUg/U/J9Hn1Jv6pKzm+oTeaG92h4M+uBnMJE0uBu1lcCtwEfAE4GrpR0cqdtGoCbgXMj4hTgM+mq7cDH07YZwLckDc/Y9caImJJ+FuRyHnaw5sY6Xtvq0fBm1jO5Xpks6Wb9VKA1IlZFxG7gXuDSTtt8ErgtIjYBRMT69M8VEbEy/f4qsB6ozbFe66GOueH94kcz64mq7jaQ9NlDrQIOe2UCjAFWZyyvAc7utM2k9Oc8BVQCfxsR/9GphqlADfBiRvNXJX0RmA3cFBG7uqnFsjB6cD/OGDec2UvX85npkwpdjpkVuZ5cmfxvYAQwpNNncA/3704V0ABcCFwJfC/zdpakY4B/A/4kItrT5puBycBZwEjgc10dWNK1kloktbS1tfVCqX1Lc2M9i9Z6NLyZda8nYfAc8LOI+FLnD/BmN/uuBcZlLI9N2zKtAWZGxJ6IeAlYQRIuSBoK/BL4fEQ807FDRKyLxC7gByS30w4SEbdHRFNENNXW+g5Ztqanc8P7xY9m1p2ehMmfAC8fYl1TN/vOAxokTZBUA1wBzOy0zc9IrkqQNJrktteqdPsHgXsi4oHMHdKrFSQJuAx4oQfnYVmaVD+YsSMGMMcvfjSzbnQbJhGxPCIONXrtxm723QtcB8wClgL3R8RiSbdKuiTdbBawQdISYC5JL60NwOXABcDVXXQB/pGkRcAiYDTwle7Ow7IniemN9fym1XPDm9nhKZdxBJJeiYjjerGevGlqaoqWlpZCl1Fyfr2ijY/f+Szfv6qJ5vS2l5n1HZLmR0R3d6FyfoCuHPe3Ine2R8ObWQ/0pGvwyEOtwmFS9vpVVXLBpFrmLHudiHeSPKYyMztQt2ECzAeCroNjd++WY8WoubGeR154jRfWbuXUscMKXY6ZFaFuwyQiJvTkQJJOiYjFuZdkxWbaSbX7R8M7TMysK735osd/68VjWREZNbgf7zpuhOeGN7ND6s0w8c30MtbcWMcLa7fy2haPhjezg/VmmPhd5WVs/2h4X52YWRc8OZb1SEPdYMaNHMAcdxE2sy70Zpi4Z1cZk0Tz5GQ0/I7dHg1vZgfqcZhIOlfSoPT7H0v6hqTjO9ZHxDn5KNCKx/TGenbtbeepVs8Nb2YHyubK5F+A7ZJOB/6KZG6Re/JSlRWlqRNGMrhflZ+bmNlBsgmTvZG8yOtS4DsRcRvJvCbWR9RUVXDBpNHMXrqe9nb3tzCzt2UTJm9Kuhn4Y+CXkiqA6vyUZcWqeXI969/cxQuvbil0KWZWRLIJk48Au4BPRMRrJBNdfT0vVVnRmja5jgrhFz+a2QGyujIB/jEinpQ0CZgC/CQ/ZVmxGjmoJhkNv9TPTczsbdmEya+BfpLGAL8CPgbclY+irLg1N9az+NWtrNuyo9ClmFmRyCZMFBHbgT8C/jkiPgy8s9udpBmSlktqlXTTIba5XNISSYsl/Tij/SpJK9PPVRntZ0palB7z2/J70Y+q6Y11gOeGN7O3ZRUmkt4NfBT4ZU/2l1QJ3AZ8ADgZuFLSyZ22aQBuBs6NiFOAz6TtI4FbgLOBqcAtkkaku/0L8EmgIf3MyOI8LEcn1g3muJEDfavLzPbLJkw+Q/JL/8F0HveJJHO2H85UoDUiVkXEbuBekq7FmT4J3BYRmwAiouOfu+8HHo2Ijem6R4EZko4BhkbEM2lX5XuAy7I4D8uRJJob63jqxQ1s37230OWYWRHocZhExBMRcQlwm6TBaUBc381uY4DVGctr0rZMk4BJkp6S9IykGd3sOyb9frhjWp5Nb6xn9952nmrdUOhSzKwIZPM6lVMl/RewGFgiab6kU3qhhiqSW1UXAlcC35M0vBeOi6RrJbVIamlra+uNQ1rqrPEjGdKvyre6zAzI7jbXd4HPRsTxEXEcyStVvtfNPmuBcRnLY9O2TGuAmRGxJyJeAlaQhMuh9l2bfj/cMQGIiNsjoikimmpra7sp1bKRjIavZfYyj4Y3s+zCZFBE7H9GEhGPA4O62Wce0CBpgqQa4ApgZqdtfkZyVYKk0SS3vVYBs4CLJY1IH7xfDMyKiHXAVknnpL24Pg48lMV5WC9pbqyj7c1dLFrr0fBmfV02YbJK0v+SND79fIHkl/4hRcRe4DqSYFgK3J8+vL9V0iXpZrOADZKWkDzQvzEiNkTERuDLJIE0D7g1bQP4FHAH0ErywslHsjgP6yXTTkpGw/tWl5kp6RDVgw2Tq4MvAeelTU8Cf9vRC6vYNTU1RUtLS6HLKDsf/tf/5K1d+3j4hvMLXYqZ5YGk+RHR1N12VT09YBoa3fXesj6mubGerz2yjFc37+DY4QMKXY6ZFUi3YSLp5xxmfve0u7D1UdMb6/jaI8uYvWw9Hzvn+O53MLOy1JMrk7/PexVWsk6oHczxo5LR8A4Ts76r2zCJiCeORiFWmjrmhv/hb19m++69DKzp8Z1TMysj2QxaXCRpYafPk5K+KWlUPou04ja9sY7de9v5zUrPDW/WV2XTNfgRkhc8fjT9/BxoAV7Dr6Lv086aMJIh/av8FmGzPiybexLTI+JdGcuLJD0XEe+S9Me9XZiVjurKCt6bMRq+osIzApj1NdlcmVRKmtqxIOksoDJd9Ktj+7jpjfW8sW0XCz0a3qxPyubK5BrgTkmDAQFbgU9IGgT8n3wUZ6XjvZNq94+GnzKuV97TaWYlJJtX0M+LiFNJ5n4/PSJOS9veioj781eilYIRg2poOn4kj/m5iVmflE1vrmGSvgHMBmZL+gdJw/JXmpWa5sY6lq7bytrNnhverK/J5pnJncCbwOXpZyvwg3wUZaWpubEegDl+8aNZn5NNmJwQEbekMyyuiogvARPzVZiVnhNqBzF+1EDf6jLrg7IJkx2SOt4YjKRzAd/PsP2SueHrefrFDby1yx38zPqSbMLkL0jmf/+dpJeB7wB/np+yrFQ1N9axe187v2n1aHizviSb3lwLIuJ04DTg1Ig4IyKe724/STMkLZfUKummLtZfLalN0oL0c03aPi2jbYGknZIuS9fdJemljHVTen7Klk9nje8YDe/nJmZ9SU9eQf/ZQ7QDEBHfOMy+lcBtwPtI5nqfJ2lmRCzptOl9EXFdZkM6RfCU9DgjSWZV/FXGJjdGxAPd1W9HV3VlBReeVMecZW0eDW/Wh/TkymRIN5/DmQq0pg/sdwP3ApceQZ0fAh6JiO1HsK8dZdMb63hj2y6eX7O50KWY2VHSk1fQfymH448BVmcsrwHO7mK7D0q6AFgB/I+IWN1p/RVA5yugr0r6Ism4l5siYlcOdVoveu+kWiorxOyl6znjuBGFLsfMjoJsHsDvJ+m5Xqzh58D4iDgNeBS4u9PPOgY4FZiV0XwzMBk4CxgJfO4QdV4rqUVSS1tbWy+WbIczfGANZx4/gsf83MSszziiMCF5N1dPrAXGZSyPTdv2i4gNGVcVdwBndjrG5cCDEbEnY591kdhFMnByKl2IiNsjoikimmpra3tYsvWG6Y11LHvtTdZs8p1Js77gSMPklz3cbh7QIGmCpBqS21UzMzdIrzw6XAIs7XSMK4GfdLWPkl4AlwEv9Lx0Oxr2j4Zf5gGMZn3BEYVJRHyhh9vtBa4juUW1FLg/IhZLulXSJelm10taLOl54Hrg6o79JY0nubLpPHXwjyQtAhYBo4GvHMl5WP6cUDuYCaMHeTS8WR/R41fQS3oTiE7NW0hmW/yriFjV1X4R8TDwcKe2L2Z8v5nkGUhX+/6O5CF+5/aLelq3FU7z5Druefpltu3ay+B+nhverJxlc2XyLeBGkl/uY4G/Bn5M0t33zt4vzUpdc2N9Mhrec8Oblb1swuSSiPhuRLwZEVsj4nbg/RFxH+D+n3aQpvEjGOrR8GZ9QjZhsl3S5ZIq0s/lwM50XefbX2b7R8PPXZ7MDW9m5SubMPko8DFgPfB6+v2PJQ0gechudpDmxjre2LabBR4Nb1bWevxUNH3A/oeHWP2b3inHys2Fk+rS0fCv8y6PhjcrW9lM2ztJ0mxJL6TLp0nqURdh67uGDazmrPEjmO0uwmZlLZvbXN8j6cK7ByAiFpIMQjQ7rObJ9R4Nb1bmsgmTgRHxbKc2T6dn3WpurAPw1YlZGcsmTN6QdAJpzy1JHwLW5aUqKysTawczcfQgv/jRrIxlEyafBr4LTJa0FvgMnrbXeqi5sY7frtrINs8Nb1aWsgmTtSRv6P0qyaj3R4Gr8lGUlZ+O0fBPrvBUAGblKJsweYika/Ae4FVgG/BWPoqy8tN0/AiGDahmtt8ibFaWsnn73tiImJG3SqysVVVWcOFJtcxdtp597UGl54Y3KyvZXJn8p6RT81aJlb3mxno2vLWbBas9Gt6s3GQTJucB8yUtl7RQ0iJJC/NVmJWf906qpSodDW9m5SWb21wfyFsV1icMG1DNWeNHMnvpev5mxuRCl2NmvajHVyYR8XJXn+72kzQjvZpplXRTF+uvltQmaUH6uSZj3b6M9pkZ7RMk/TY95n3plMBWApob61j++pus3ujR8Gbl5EjngO8RSZXAbSRXNScDV0o6uYtN74uIKennjoz2HRntl2S0/x3wzYg4EdgEfCJf52C9q2NueN/qMisveQ0TYCrQGhGrImI3yfiUS3M5oCQBFwEPpE13A5flVKUdNRNGD2Ji7SB3ETYrM/kOkzHA6ozlNXQxpzvwwfSh/gOSxmW095fUIukZSR2BMQrYHBEdQ6kPdUwrUtMb63lm1Qbe3Lmn0KWYWS/Jd5j0xM+B8RFxGsmo+rsz1h0fEU3Afwe+lb4brMckXZuGUUtbm0deF4vmyXXs2Rc86bnhzcpGvsNkLZB5pTE2bdsvIjZExK508Q7gzIx1a9M/VwGPA2cAG4Dhkjp6oh10zIz9b4+Ipohoqq2tzf1srFec2TEa3m8RNisb+Q6TeUBD2vuqhmT+k5mZG0g6JmPxEmBp2j5CUr/0+2jgXGBJRAQwF/hQus9VJK96sRJRVVnBtJNqmbs8GQ1vZqUvr2GSPte4DphFEhL3R8RiSbdK6uiddb2kxZKeB64Hrk7bG4GWtH0u8LWIWJKu+xzwWUmtJM9Qvp/P87De19xYz8a3drNg9aZCl2JmvSCbQYtHJCIeBh7u1PbFjO83k8zg2Hm//wS6fH1Lettrau9WakfTe09KRsM/tnQ9Zx4/stDlmFmOiuEBvPVBQ/tXM3XCSI83MSsTDhMrmObGela8vs2j4c3KgMPECmZ6Oje8p/M1K30OEyuY40cN4oTaQe4ibFYGHCZWUNMb6/ntSx4Nb1bqHCZWUM2N9ezZF/x6hUfDm5Uyh4kV1LuOG87wgdXu1WVW4hwmVlDJaPg6j4Y3K3EOEyu45sY6Nm3fw3+94tHwZqXKYWIFd8Gkt0fDm1lpcphYwQ3tX83ZEz0a3qyUOUysKDRPrmfl+m28ssGj4c1KkcPEisL0dG54j4Y3K00OEysKx40aSEPdYGYvc5iYlSKHiRWNixrr+O2qjWz1aHizkpP3MJE0Q9JySa2Sbupi/dWS2iQtSD/XpO1TJD2dTpy1UNJHMva5S9JLGftMyfd5WP5Nb6xnb3vw6xVthS7FzLKU18mxJFUCtwHvA9YA8yTNzJgxscN9EXFdp7btwMcjYqWkY4H5kmZFxOZ0/Y0R8UA+67ej613HjWDEwGRu+D847dhCl2NmWcj3lclUoDUiVkXEbuBe4NKe7BgRKyJiZfr9VWA9UJu3Sq3gKivk0fBmJSrfYTIGWJ2xvCZt6+yD6a2sBySN67xS0lSgBngxo/mr6T7flNSvV6u2gmlurGfz9j0859HwZiWlGB7A/xwYHxGnAY8Cd2eulHQM8G/An0REe9p8MzAZOAsYCXyuqwNLulZSi6SWtjbfhy8FF0waTXWl3EXYrMTkO0zWAplXGmPTtv0iYkNE7EoX7wDO7FgnaSjwS+DzEfFMxj7rIrEL+AHJ7bSDRMTtEdEUEU21tb5DVgqG9K/m7AmjPGGWWYnJd5jMAxokTZBUA1wBzMzcIL3y6HAJsDRtrwEeBO7p/KC9Yx9JAi4DXsjbGdhR19xYR+v6bby84a1Cl2JmPZTXMImIvcB1wCySkLg/IhZLulXSJelm16fdf58HrgeuTtsvBy4Aru6iC/CPJC0CFgGjga/k8zzs6Hp7NLyvTsxKhSL6Rq+ZpqamaGlpKXQZ1kMXf/MJRg/ux48/eU6hSzHr0yTNj4im7rYrhgfwZgdpbqzn2Zc8Gt6sVDhMrChNb6xjb3vwxHL3wjMrBQ4TK0pTxo1g5KAaz3FiViIcJlaUKivEhSfVMnd5G3v3tXe/g5kVlMPEitb0xnq27NjDc69s7n5jMysoh4kVrfMbktHwvtVlVvwcJla0hvSv5pyJo/xqFbMS4DCxotY8uY4X297id294NLxZMXOYWFFr9tzwZiXBYWJFbdzIgZxUP8QvfjQrcg4TK3rNjXXM+91GtuzwaHizYuUwsaLXnM4N/4TnhjcrWg4TK3pTxg1nlEfDmxU1h4kVvWQ0fB2PezS8WdFymFhJmN5Yx5Yde5j/sueGNytGDhMrCedPqqWmsoLZy9yry6wY5T1MJM2QtFxSq6Sbulh/taS2jNkUr8lYd5Wklennqoz2MyUtSo/57XT6Xitjg/tVcfbEkR5vYlak8homkiqB24APACcDV0o6uYtN74uIKennjnTfkcAtwNnAVOAWSSPS7f8F+CTQkH5m5PM8rDhMb6xnVdtbvOTR8GZFJ99XJlOB1ohYFRG7gXuBS3u47/uBRyNiY0RsAh4FZkg6BhgaEc9EMufwPcBl+SjeiktzYx2Ae3WZFaF8h8kYYHXG8pr3gTi9AAAJEElEQVS0rbMPSloo6QFJ47rZd0z6vbtjWpkZO2Igk98xxLe6zIpQMTyA/zkwPiJOI7n6uLu3DizpWkktklra2jzgrRwko+E3sWW7R8ObFZN8h8laYFzG8ti0bb+I2BARu9LFO4Azu9l3bfr9kMfMOPbtEdEUEU21tbVHfBJWPJob69nXHjy+wr26zIpJvsNkHtAgaYKkGuAKYGbmBukzkA6XAEvT77OAiyWNSB+8XwzMioh1wFZJ56S9uD4OPJTn87AiMWXscEYPrvGLH82KTFU+Dx4ReyVdRxIMlcCdEbFY0q1AS0TMBK6XdAmwF9gIXJ3uu1HSl0kCCeDWiNiYfv8UcBcwAHgk/VgfUFEhpp1Ux6zFr3H/vNUMH1jNiEE1DB9QzfCBNQwfWE11ZTHcvTXrW5R0iCp/TU1N0dLSUugyrBc8ubKNq+58lvZD/NUd0q+KYQOrGZGGy4iBNYwYWM2w9M+O9uHp8vCBNQztX4WHK5kdTNL8iGjqbru8XpmY5cP5DbW88KX3s2Hbbrbs2MOm7bvZtH0Pm7fvZvP2ZLnjz03b97B643Y2bd/D1p17ONS/nSorxLAB1QeEz/CByRXPiEFvh1LHFdCIQcly/+rKo3vyZkXKYWIlaWBNFQNHVh3QQ6M7+9qDLTuS0OkIn0OF0Kubd7Lk1a1s2r6HHXv2HfKY/asrGD4g4wpoUDXDBhz6CmjEwGqGDaimqkhuxbW3B+0RtAe0RxDpn/siiHbSdcn6SNvbI9mvY9sDt3n72MHbCwe092Cbznp03AO2jy7bD/453R+rHJwxbnje/845TKzPqKwQIwfVMHJQTVb77dyzj83b97B5x242vZURQjvS8HkrWd6yYzcrXt+2P5z2Huo+HDCkf9X+K6ABNZX7f1m37/8Fnf7ybo+MdRnr2w8OgPYI9rUfvG0E6XE6B0Cu/4taqXj+losZNsBhYlZQ/asrecewSt4xrH+P94kI3ty1ly3bu78Nt3P3PiSoqqygQkKCComKjj8rMr5nrK+sOHhbSVRWHLxthUiPc+C2FYLK9Gcc9HMz9lNGe2XmcSsOPFZFp+dOmUsHrlKX7Qdu37NjHbh/1wfr/DQs89iHPO5Be5WugTX5vx3rMDHLA0kM7V/N0P7VjBs5sNDlmOVdcdy4NTOzkuYwMTOznDlMzMwsZw4TMzPLmcPEzMxy5jAxM7OcOUzMzCxnDhMzM8tZn3lrsKQ24OUj3H008EYvllNI5XIu5XIe4HMpVuVyLrmex/ER0e3sgn0mTHIhqaUnr2AuBeVyLuVyHuBzKVblci5H6zx8m8vMzHLmMDEzs5w5THrm9kIX0IvK5VzK5TzA51KsyuVcjsp5+JmJmZnlzFcmZmaWM4dJD0n6sqSFkhZI+pWkYwtd05GQ9HVJy9JzeVDS8ELXdKQkfVjSYkntkkqy142kGZKWS2qVdFOh6zlSku6UtF7SC4WuJReSxkmaK2lJ+nfrhkLXdKQk9Zf0rKTn03P5Ul5/nm9z9YykoRGxNf1+PXByRPx5gcvKmqSLgTkRsVfS3wFExOcKXNYRkdQItAPfBf46IloKXFJWJFUCK4D3AWuAecCVEbGkoIUdAUkXANuAeyLinYWu50hJOgY4JiKekzQEmA9cVqL/TQQMiohtkqqB3wA3RMQz+fh5vjLpoY4gSQ0CSjKFI+JXEbE3XXwGGFvIenIREUsjYnmh68jBVKA1IlZFxG7gXuDSAtd0RCLi18DGQteRq4hYFxHPpd/fBJYCYwpb1ZGJxLZ0sTr95O33lsMkC5K+Kmk18FHgi4Wupxf8KfBIoYvow8YAqzOW11Civ7jKkaTxwBnAbwtbyZGTVClpAbAeeDQi8nYuDpMMkh6T9EIXn0sBIuLzETEO+BFwXWGrPbTuziPd5vPAXpJzKVo9ORez3iZpMPBT4DOd7kqUlIjYFxFTSO5ATJWUt1uQVfk6cCmKiOk93PRHwMPALXks54h1dx6Srgb+AGiOIn9olsV/k1K0FhiXsTw2bbMCSp8v/BT4UUT8v0LX0xsiYrOkucAMIC+dJHxl0kOSGjIWLwWWFaqWXEiaAfwNcElEbC90PX3cPKBB0gRJNcAVwMwC19SnpQ+tvw8sjYhvFLqeXEiq7eitKWkASUePvP3ecm+uHpL0U+Akkt5DLwN/HhEl969ISa1AP2BD2vRMKfZKA5D034B/AmqBzcCCiHh/YavKjqTfA74FVAJ3RsRXC1zSEZH0E+BCkjfUvg7cEhHfL2hRR0DSecCTwCKS/68D/M+IeLhwVR0ZSacBd5P83aoA7o+IW/P28xwmZmaWK9/mMjOznDlMzMwsZw4TMzPLmcPEzMxy5jAxM7OcOUzMzCxnDhOzHpK0rfutDrv/A5Impt8HS/qupBclzZf0uKSzs/nZki6Q9JykvZI+lNFeK+k/cqnVLFsOE7OjQNIpQGVErEqb7iB5y25DRJwJ/AnJgL9svAJcDfw4szEi2oB1ks7NqWizLDhMzLKkxNfTF04ukvSRtL1C0j+nk489KunhjCuGjwIPpdudAJwNfCEi2gEi4qWI+GU2dUTE7yJiIW+P1M70s/Rnmh0VDhOz7P0RMAU4HZgOfD2dVOmPgPHAycDHgHdn7HMuyURLAKeQvPplXx5rbAHOz+PxzQ7gMDHL3nnAT9LXe78OPAGclbb/e0S0R8RrwNyMfY4B2o5ijeuBkpxa2kqTw8Ts6NgB9E+/LwZOT6ftzZf+6c80OyocJmbZexL4SDqLXS1wAfAs8BTwwfTZST3JW3Q7LAVOBIiIF0luQ30pfeU5ksZL+v1erHESeZq3wqwrDhOz7D0ILASeB+YAf5Pe1vopydS7S4AfAs8BW9J9fsmB4XINUA+0SnoBuIvk1tShDJS0JuPzWUlnSVoDfBj4rqTFGdtPS3+m2VHhV9Cb9SJJgyNim6RRJFcr50bEa+nkRHPT5Xw+eO+o49fApRGxKd8/yww8ba9Zb/tFOrtdDfDl9IqFiNgh6RZgDMn4kLxJb719w0FiR5OvTMyKRHo1M7uLVc0RsaGLdrOi4TAxM7Oc+QG8mZnlzGFiZmY5c5iYmVnOHCZmZpYzh4mZmeXs/wMlzUh9osL/XAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "n_Cs = len(Cs)\n",
    "n_classes = 5\n",
    "scores=  np.zeros((n_classes,n_Cs))\n",
    "mse_mean_L1= -np.mean(ls_L1.scores_[1],axis = 0)\n",
    "\n",
    "plt.plot(np.log10(Cs), mse_mean_L1.reshape(n_Cs,1)) \n",
    "\n",
    "plt.xlabel('log(C_L1)')\n",
    "plt.ylabel('neg-logloss_L1')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.69314718, 0.66752303, 0.49510891, 0.49131283, 0.4917517 ,\n",
       "       0.49181307, 0.49181921])"
      ]
     },
     "execution_count": 231,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mse_mean_L1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 242,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 0.491312833954765\n"
     ]
    }
   ],
   "source": [
    "best_C_L1 = np.argmin(mse_mean_L1)\n",
    "best_score_L1 = np.min(mse_mean_L1)\n",
    "print(Cs[best_C_L1], best_score_L1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 233,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{1: array([[-0.61403412, -0.54243089, -0.52834518, -0.5335345 , -0.5345814 ,\n",
       "         -0.53469487, -0.53470632],\n",
       "        [-0.60085861, -0.49233919, -0.43646534, -0.42797809, -0.4271889 ,\n",
       "         -0.42711176, -0.42710411],\n",
       "        [-0.6085931 , -0.51871522, -0.4799827 , -0.4796303 , -0.4800732 ,\n",
       "         -0.48012597, -0.4801313 ],\n",
       "        [-0.61276958, -0.54684635, -0.5414634 , -0.55175937, -0.55371125,\n",
       "         -0.55392256, -0.55394388],\n",
       "        [-0.60440094, -0.508207  , -0.468622  , -0.46369361, -0.46325949,\n",
       "         -0.46321731, -0.46321305]])}"
      ]
     },
     "execution_count": 233,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ls_L2.scores_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 238,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAELCAYAAADKjLEqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3XmcVPWd7vHPU9XdbLIpLVKCgIILCk20BRNNNHHDRMpJnGgcjZponGTGl9lnzE3mOjE3M0lMcjOLmRu3jFmMccwiuESJY4yZRKVRQAFRUCOLQAsBMSBNd3/vH33Atm3oqqaqT3X383696kWdU79T9VQw9XB2RQRmZmZdyaQdwMzMegcXhpmZFcSFYWZmBXFhmJlZQVwYZmZWEBeGmZkVxIVhZmYFcWGYmVlBXBhmZlaQqrQDlNKoUaNiwoQJaccwM+tVFixY8EpE1HY1rk8VxoQJE2hoaEg7hplZryLpj4WM8yYpMzMriAvDzMwK4sIwM7OCuDDMzKwgLgwzMyuIC8PMzAriwjAzs4K4MIDN25q45Xcv0Lh1R9pRzMwqlgsDaNy6g2vvXsq9T72cdhQzs4rlwgAmjx7KkQcN5a6Fa9KOYmZWsVwYifz0HE+8tJlVm7alHcXMrCK5MBKzp+UAmLt4bcpJzMwqkwsjMW7/wRx7yAjmLHRhmJl1xoXRTr4uxzPrtvLc+q1pRzEzqzgujHbeNy1HRjBnkdcyzMw6cmG0Uzt0AO84bBRzFq0lItKOY2ZWUVwYHeTrcvxx4zYWr96SdhQzs4riwujgzGMOoiab8WYpM7MOyl4YkmZJWi5phaSr9zDmPElLJS2RdFu7+ZdIei55XFLurADDB1Vz8hG13L14LS2t3ixlZrZLWQtDUha4HjgLmAJcIGlKhzGTgS8AJ0bE0cCnkvn7A9cAM4EZwDWSRpYz7y75uhzrX93B4y9s6omPMzPrFcq9hjEDWBERz0dEE3A7cE6HMR8Dro+IPwFExIZk/pnAvIjYlLw2D5hV5rwAnHbUaAbXZL1ZysysnXIXxsHAqnbTq5N57R0OHC7pfyQ9KmlWEcuWxaCaLKdPGc19T79MU3NrT3ykmVnFq4Sd3lXAZOAU4ALgRkkjCl1Y0hWSGiQ1NDY2lixUvi7H5m07+d2K0r2nmVlvVu7CWAOMazc9NpnX3mpgTkTsjIgXgGdpK5BCliUiboiI+oior62tLVnwd06uZcTgal8qxMwsUe7CmA9MljRRUg3wIWBOhzG/pG3tAkmjaNtE9TxwP3CGpJHJzu4zknk9oqYqw1nHjOGBpevZ3tTSUx9rZlaxyloYEdEMXEnbD/0y4I6IWCLpWkn5ZNj9wEZJS4GHgM9HxMaI2AR8hbbSmQ9cm8zrMfm6HNuaWnjwmfU9+bFmZhVJfekSGPX19dHQ0FCy92tpDd7xtQepGzuCGy6uL9n7mplVEkkLIqLLH7lK2OldsbIZcfa0HL9Z3siW7TvTjmNmlioXRhfydTmaWlq5/+l1aUcxM0uVC6ML08YOZ/wBg30Sn5n1ey6MLkgiX5fj9ytfYcPW19OOY2aWGhdGAfJ1OVoD7l38ctpRzMxS48IowOTRQznyoKHeLGVm/ZoLo0D56TmeeGkzqzZtSzuKmVkqXBgFmj0tB8DcxV7LMLP+yYVRoHH7D+bYQ0b42lJm1m+5MIqQr8vxzLqtPLd+a9pRzMx6nAujCO+bliMjvPPbzPolF0YRaocO4B2HjWLOorX0pWtwmZkVwoVRpHxdjj9u3Mbi1VvSjmJm1qNcGEU685iDqMlmvFnKzPodF0aRhg+q5pQjarl78VpaWr1Zysz6DxdGN+Sn51j/6g4ef6FH7+dkZpYqF0Y3nHrkaIbUZL1Zysz6FRdGNwyqyXL6lNHc9/TLNDW3ph3HzKxHuDC6KT89x+ZtO/ndisa0o5iZ9QgXRjedNKmWEYOrucuXCjGzfsKF0U01VRnOOmYM85auZ3tTS9pxzMzKzoWxD/J1ObY1tfDrZevTjmJmVnYujH0wY+L+jB42wEdLmVm/4MLYB9mMOHtajoeXN7Jl+86045iZlZULYx/l63I0tbRy/9Pr0o5iZlZWZS8MSbMkLZe0QtLVnbx+qaRGSQuTx+XtXvuGpCWSlkn6V0kqd95iTRs7nPEHDPZmKTPr88paGJKywPXAWcAU4AJJUzoZ+tOImJ48bkqWfQdwIjANOAY4Hji5nHm7QxL5uhy/X/kKG7a+nnYcM7OyKfcaxgxgRUQ8HxFNwO3AOQUuG8BAoAYYAFQDFXk4Ur4uR2vAvYtfTjuKmVnZlLswDgZWtZtenczr6FxJiyXdKWkcQET8AXgIeDl53B8Ry8qct1smjx7KkQcN9WYpM+vTKmGn91xgQkRMA+YBtwJImgQcBYylrWTeI+mdHReWdIWkBkkNjY3pXaYjPz3HEy9tZtWmballMDMrp3IXxhpgXLvpscm83SJiY0TsSCZvAo5Lnr8feDQiXouI14D7gLd3/ICIuCEi6iOivra2tuRfoFCzp+UAmLvYaxlm1jeVuzDmA5MlTZRUA3wImNN+gKQx7SbzwK7NTi8BJ0uqklRN2w7vitwkBTBu/8Ece8gI5vjaUmbWR5W1MCKiGbgSuJ+2H/s7ImKJpGsl5ZNhVyWHzi4CrgIuTebfCawEngIWAYsiYm458+6rfF2OZ9Zt5bn1W9OOYmZWcoroO7cZra+vj4aGhtQ+v3HrDmb+06/523dP4rNnHJFaDjOzYkhaEBH1XY2rhJ3efUbt0AGcOGkUcxatpS8VsZkZuDBKbnZdjj9u3Mbi1VvSjmJmVlIujBI78+iDqMlmfE6GmfU5LowSGz6omlOOqGXuorW0tHqzlJn1HS6MMshPz7Fh6w4ee2Fj2lHMzErGhVEGpx45miE1WeZ6s5SZ9SEujDIYVJPl9CmjufepdTQ1t6Ydx8ysJFwYZZKfnmPL9p088lx617cyMyslF0aZnDSplhGDq320lJn1GS6MMqmpynDWMWOYt3Q925ta0o5jZrbPXBhllK/Lsa2phV8vq8j7PpmZFcWFUUYzJu7P6GEDvFnKzPqELgtD0lRJj0paJekGSSPbvfZ4eeP1btmMOHtajoeXN7Jl+86045iZ7ZNC1jD+A/hHYCrwLPA7SYclr1WXKVefka/L0dTSyv1Pr0s7ipnZPimkMIZGxK8iYnNEfJO2+1v8StIJgK990YVpY4cz/oDB3ixlZr1eQfswJA3f9TwiHgLOBX4IjC9Trj5DEvm6HL9f+Qobtr6edhwzs24rpDC+DhzVfkZELAZOBX5ejlB9Tb4uR2vAvYtfTjuKmVm3dVkYEXFbRDzayfyXAN/0oQCTRw/lqDHDvFnKzHq1fT2s9rySpOgH8nU5nnhpM6s2bUs7iplZt+xrYagkKfqB2XVjAJi72GsZZtY7FXIexv57eByAC6NgY0cO5rjxI5mz0IVhZr1TVQFjFtB2+Gxn5dBU2jh9W74uxzVzlvDs+q0cPnpo2nHMzIpSyE7viRFxaPJnx8ehu8ZJOrq8UXu/904dQ0Z4LcPMeqVSXkvqhyV8rz6pdugATpw0ijmL1hLhcx7NrHcpZWF4f0YBZtfleGnTNhat9hHJZta7lLIwOv0ns6RZkpZLWiHp6k5ev1RSo6SFyePydq8dIukBScskLZU0oYR5U3Hm0QdRk814s5SZ9Tplvby5pCxwPXAWMAW4QNKUTob+NCKmJ4+b2s3/AXBdRBwFzAA2lDNvTxg+qJpTjqjl7sVraWn1Zikz6z1KWRidHTE1A1gREc9HRBNwO3BOIW+WFEtVRMwDiIjXIqJPnPWWn55jw9YdPPbCxrSjmJkVrODCkHSipCHJ84skfVvS7osPRsQJnSx2MLCq3fTqZF5H50paLOlOSeOSeYcDmyX9XNKTkq5L1lh6vVOPHM2QmixzfakQM+tFilnD+A9gm6Q64LPASto2Ge2rucCEiJgGzANuTeZXAe8EPgccDxwKXNpxYUlXSGqQ1NDY2FiCOOU3qCbL6VNGc+9T62hqbk07jplZQYopjOZoOxb0HODfI+J6oKuzz9YA49pNj03m7RYRGyNiRzJ5E3Bc8nw1sDDZnNUM/BI4tuMHRMQNEVEfEfW1tbVFfJ105afn2LJ9J4881ztKzsysmMLYKukLwEXAPZIydH3HvfnAZEkTJdUAHwLmtB8gaUy7yTywrN2yIyTtaoH3AEuLyFvRTppUy4jB1b6CrZn1GsUUxvnADuCyiFhH29rCdXtbIFkzuBK4n7YiuCMilki6VlI+GXaVpCWSFgFXkWx2iogW2jZHPSjpKdrO87ixiLwVraYqw1nHjGHe0vVsb2pJO46ZWZdU6BnHyQ7v1yOiRdLhwJHAfRGxs5wBi1FfXx8NDQ1pxyjYH1Zu5IIbH+XfLngbs+tyaccxs35K0oKIqO9qXDFrGL8FBkg6GHgA+DDwn92LZwAzJu7P6GEDvFnKzHqFYgpDyXkQHwC+GxEfBI4pT6z+IZsRs6fleHh5I1u2V8yKmplZp4oqDElvBy4E7unG8taJ/PQcTS2t3P/0urSjmJntVTE/+J8CvgD8ItlxfSjwUHli9R9TDx7OhAMGe7OUmVW8ggsjIh6OiDxwvaT9kvMjripjtn5BEvm6HL9f+Qobtr6edhwzsz0q5tIgUyU9CSwBlkpa4JsmlUZ+eo7WgHsXv5x2FDOzPSpmk9T3gM9ExPiIOIS2y4P0mfMi0jTpwKEcNWYYd3mzlJlVsGIKY0hE7N5nERG/AYaUPFE/la/L8eRLm1m1qU9ckNfM+qBiCuN5Sf8gaULy+BLwfLmC9Tez69qukOKd32ZWqYopjI8CtcDPk0dtMs9KYOzIwRw3fqQveW5mFauYo6T+FBFXRcSxyeOTEfGncobrb/J1OZ5Zt5Vn129NO4qZ2VtUdTVA0lz2cL9ugORQWyuB904dw5fnLmHOwrV87swj0o5jZvYmXRYG8M2ypzAAaocO4MRJo5izaC2fPeNwJKUdycxsty4LIyIe7okg1mZ2XY6/u3Mxi1ZvYfq4EWnHMTPbrZgT955K7rvd/vGIpP8r6YByhuxPzjz6IGqyGeYs9M5vM6ssxRwldR9tFx28MHnMBRqAdfgy5yUzfFA1pxxRy92L19LSWti9SszMekIh+zB2OS0i2t9T+ylJT0TEsZIuKnWw/iw/PccDS9fz2Asbecdho9KOY2YGFLeGkZU0Y9eEpOOBbDLZXNJU/dypR45mSE3W52SYWUUppjAuB26W9IKkF4GbgcuTW7f+cznC9VeDarKcPmU09z61jqbm1rTjmJkBxZ24Nz8ipgLTgbqImJbM+3NE3FG+iP3TOdMPZsv2nTzyXGPaUczMgOKOkhou6dvAg8CDkr4laXj5ovVvJ00excjB1b62lJlVjGI2Sd0CbAXOSx6vAt8vRyiD6myGs6aOYd7S9Wxvakk7jplZUYVxWERck9xp7/mI+DJwaLmCWdu1pbY1tfDrZevTjmJmVlRhbJd00q4JSScC20sfyXaZMWF/Dho20JulzKwiFHMexieAW5P9FgI2AZeWI5S1yWTE2dPGcOsfXmTLtp0MH1yddiQz68eKOUpqYUTUAdOAqRHxtohY1NVykmZJWi5phaSrO3n9UkmNkhYmj8s7vD5M0mpJ/15o1r4kPz3HzpbgV0t8v28zS1chlzf/zB7mAxAR397LslngeuB0YDUwX9KciFjaYehPI+LKPbzNV4DfdpWzr5p68HAmHDCYOYvWcv7xh6Qdx8z6sULWMIZ28dibGcCKZCd5E3A7cE6h4SQdB4wGHih0mb5GEvm6HH9YuZENW19PO46Z9WOFXN78y/vw/gcDq9pNrwZmdjLuXEnvAp4FPh0RqyRlgG8BFwGn7ekDJF0BXAFwyCF981/g+ek5/vW/V3DP4pf5yIkT045jZv1UMUdJ7SbpiRJmmAtMiIhpwDzg1mT+3wD3RsTqvS0cETdERH1E1NfW1pYwVuWYdOBQjhozzEdLmVmqulUYtB0lVYg1wLh202OTebtFxMaI2JFM3gQclzx/O3Blct2qbwIXS/paN/P2evm6HE++tJlVm7alHcXM+qnuFsY9BY6bD0yWNFFSDfAhYE77AZLGtJvMA8sAIuLCiDgkIiYAnwN+EBFvOcqqv5hd1/Y/k9cyzCwt3SqMiPhSgeOagSuB+2krgjsiYomkayXlk2FXSVoiaRFwFT63o1NjRw7muPEjfclzM0uNIgq7q5ukrUDHwVtou+veZyPi+RJnK1p9fX00NDSkHaNsbv39i1wzZwkPfPpdHD66qwPUzMwKI2lBRNR3Na6YNYzvAJ+n7cinsbRtJrqNtkNlb+lOSCvOe6eOISN8v28zS0UxhZGPiO9FxNaIeDUibgDOjIifAiPLlM/aqR06gBMnjWLOorUUumZoZlYqxRTGNknnScokj/OAXWeS+derh8yuy/HSpm0sWr0l7Shm1s8UUxgXAh8GNgDrk+cXSRpE245t6wGzjjmImqqMN0uZWY8r5uKDz0fE7IgYFRG1yfMVEbE9In5XzpD2hmEDq3n3EbXcvXgtLa1esTOznlPMLVoPl/SgpKeT6WmSCjq81korX3cwG7bu4LEXNqYdxcz6kWI2Sd0IfAHYCRARi2k7Ec962KlHHciQmqzPyTCzHlVMYQyOiMc7zGsuZRgrzMDqLGccfRD3PrWOpubWtOOYWT9RTGG8IukwkiOiJP0l4Lv6pCRfl2PL9p088lxj2lHMrJ8opjD+FvgecKSkNcCngI+XJZV16aTJoxg5uJq7fLSUmfWQYu7pvQb4PvAQsD/wKnAJcG0ZclkXqrMZzpo6hl88sYZtTc0Mrinmr9LMrHjFrGHcBcymbaf3WuA14M/lCGWFydfl2L6zhV8v25B2FDPrB4r5Z+nYiJhVtiRWtBkT9uegYQOZs3At+bpc2nHMrI8rZg3j95Kmli2JFS2TEWdPG8PDz25gy7adaccxsz6umMI4CVggabmkxZKekrS4XMGsMPnpOXa2BL9a4gPWzKy8itkkdVbZUli3TT14OBMOGMycRWs5//hD0o5jZn1YwYUREX8sZxDrHknk63L8+0Mr2LD1dQ4cOjDtSGbWR3X3nt5WQfLTc7QG3LPYm6XMrHxcGH3ApAOHctSYYczxtaXMrIxcGH1Evi7Hky9tZtWmbWlHMbM+yoXRR8yuGwPgtQwzKxsXRh8xduRg6seP9CXPzaxsXBh9SH56jmfWbeXZ9VvTjmJmfZALow9579QxZDPy/b7NrCzKXhiSZiVnh6+QdHUnr18qqVHSwuRxeTJ/uqQ/SFqSnFl+frmz9naj9hvAOw47gLsWrWFni2+sZGalVdbCkJQFrqftLPEpwAWSpnQy9KcRMT153JTM2wZcHBFHA7OA70gaUc68fcGHTxjPqk3b+eTtT9Ls0jCzEir3GsYMYEVEPB8RTcDtwDmFLBgRz0bEc8nztcAGoLZsSfuIM44+iH84ewr3PrWOT9+xiJbWSDuSmfUR5b7rzsHAqnbTq4GZnYw7V9K7gGeBT0dE+2WQNAOoAVaWK2hfctlJE2luaeWf73uG6oy47oN1ZDNKO5aZ9XKVcJu2ucBPImKHpL8GbgXes+tFSWOAHwKXRMRbtrFIugK4AuCQQ3zxvV3++uTDaG4Nrrt/OVVZ8bUPTCPj0jCzfVDuwlgDjGs3PTaZt1tEbGw3eRPwjV0TkoYB9wBfjIhHO/uAiLgBuAGgvr7e21/a+dt3T6KpuZV/efA5qrIZvvoXxyC5NMyse8pdGPOByZIm0lYUHwL+qv0ASWMiYtdV8/LAsmR+DfAL4AcRcWeZc/ZZnzptMjtbWvnub1ZSnRH/mD/apWFm3VLWwoiIZklXAvcDWeCWiFgi6VqgISLmAFdJygPNwCbg0mTx84B3AQdI2jXv0ohYWM7MfY0kPn/mETS3Bjf89nmqshm+9L6jXBpmVjRF9J2tOPX19dHQ0JB2jIoUEVx791K+/z8v8vGTD+PvZx3h0jAzACQtiIj6rsZVwk5v6wGS+N9nT2FnSyv/7+GV1GTFZ844Iu1YZtaLuDD6EUlcmz+G5pbgX/97BVXZDFedOjntWGbWS7gw+plMRvzT+6fS3Bp8e96zVGXF35wyKe1YZtYLuDD6oUxGfP3caexsaeUbv1pOTTbD5e88NO1YZlbhXBj9VDYjvvXBOppbg/9zzzKqMuLSEyemHcvMKpgLox+rymb4zvnTaW5p5R/nLqUqm+GiE8anHcvMKpTvh9HPVWcz/NsFx3LaUQfypV8+zU/nv5R2JDOrUC4Mo6Yqw/UXHsvJh9dy9c+f4mcLVqcdycwqkAvDABhQleV7Hz6OEw8bxefvXMRdC9d0vZCZ9SsuDNttYHWWGy+uZ8bE/fnMHYu4Z/HLXS9kZv2GC8PeZFBNlpsvOZ5jDxnBJ29/kvuXrEs7kplVCBeGvcWQAVV8/yMzmDp2OFfe9gQPLlufdiQzqwAuDOvUfgOquPWjMzhqzDA+8aMn+M3yDWlHMrOUuTBsj4YNrOaHH53J5NH7ccUPF/C7515JO5KZpciFYXs1fHA1P7psJoeOGsLlP5jPo89v7HohM+uTXBjWpZFDavjR5TMZN3IwH/3P+cx/cVPakcwsBS4MK8io/Qbw44/N5KDhA/nI9+fzxEt/SjuSmfUwF4YV7MChA/nJx05g1H41XHLz4yxevTntSGbWg1wYVpTRwwZy28dOYMSQai666TGeXrMl7Uhm1kNcGFa03IhB3Hb5CQwdWM2Hb36MZ9a9mnYkM+sBLgzrlnH7D+a2j81kQFWWC298jOfWb007kpmVmQvDum38AUP4yRUnkM2IC258jJWNr6UdyczKyIVh+2TiqCHc9rETgOCvbnyUF1/5c9qRzKxMXBi2zyYduB8/vvwEdra0lcaqTdvSjmRmZeDCsJI44qCh/Oiymfy5qYULbnyUNZu3px3JzEqs7IUhaZak5ZJWSLq6k9cvldQoaWHyuLzda5dIei55XFLurLZvpuSG8aPLZrJl+04uuOFR1m15Pe1IZlZCZS0MSVngeuAsYApwgaQpnQz9aURMTx43JcvuD1wDzARmANdIGlnOvLbvpo4dzg8vm8mmPzdxwY2PsuFVl4ZZX1HuNYwZwIqIeD4imoDbgXMKXPZMYF5EbIqIPwHzgFllymklNH3cCG796PFsePV1LrjxURq37kg7kpmVQLkL42BgVbvp1cm8js6VtFjSnZLGFbmsVaDjxu/PLZcez9rNr3PRTY+x6c9NaUcys31UCTu95wITImIabWsRtxazsKQrJDVIamhsbCxLQOuemYcewM2X1PPixj9z4U2PsXmbS8OsNyt3YawBxrWbHpvM2y0iNkbErm0WNwHHFbpssvwNEVEfEfW1tbUlC26l8Y5Jo7jx4npWNr7Gh29+nC3bd6Ydycy6qdyFMR+YLGmipBrgQ8Cc9gMkjWk3mQeWJc/vB86QNDLZ2X1GMs96mXcdXsv3LjqOZ9a9ysW3PM7W110aZr1RWQsjIpqBK2n7oV8G3BERSyRdKymfDLtK0hJJi4CrgEuTZTcBX6GtdOYD1ybzrBd695EH8t0Lj2PJmi1c+v35/HlHc9qRzKxIioi0M5RMfX19NDQ0pB3D9uK+p17myp88yXHjR/KfHzmewTVVaUcy6/ckLYiI+q7GVcJOb+tHzpo6hu+cP52GFzdx+a0NvL6zJe1IZlYgF4b1uNl1Ob51Xh1/eH4jH/uBS8Ost3BhWCre/7axfP3caTzy3Ct84kcL2NHs0jCrdC4MS8159eP4p/dP5aHljVx525PsbGlNO5KZ7YULw1L1VzMP4dpzjmbe0vVc9ZMnaXZpmFUsF4al7uK3T+Afzp7CfU+v49N3LHJpmFUoH9NoFeGykybS3NLKP9/3DNUZcd0H68hmlHYsM2vHhWEV469PPoydLa1884FnqcqKr31gGhmXRqoigtaAltagpTVobm2lpTWS15IxHca3n/fGmLcO3tOYvb5vh9PG3vL+nSzfMdOb36f856H11KluQwZUkRsxqKyf4cKwinLleyazsyX4lwefoyqb4at/cQxS6Uuj/Q9ha7T9GLZE0Nr6xvNo90O5a0zbn3tfrrWVtj/bzd+13K4f3OZd43f/EActra3Je7fu4fU3j93bmPY/7nt+vf24PX+u9Q6nTxnNjRd3ee7dPnFhWMX51GmT2dnSynd/s5LFqzczuLqKljf9YHf8Ee/8h313ISQ/5B2XqXQSVGVENiOqMhkygqpsJpkWGYmqrHZPZzMZshnIZjJvWm5gtTpZJvPGdEZks9q9TFbtpzNkO3xORmJXh2t3Vr0pd/vXds1Qh9fb5qnTZd6YftPgTj+z0/ftsHxn/+bouHw5leHfPG8xetjAsn+GC8MqjiQ+f+YRDB1YzUPPbCCTgepMhoze+EHL7Pphy+x6zlvmZcQexraf12G5ApfPZnjT2I7LZzNt36P9/I4/um9MZ5Jl3vyj7c1xVmlcGFaRJPGJUw7jE6cclnYUM0v4sFozMyuIC8PMzAriwjAzs4K4MMzMrCAuDDMzK4gLw8zMCuLCMDOzgrgwzMysIIqeujJWD5DUCPxxH95iFPBKieKkqa98D/B3qUR95XuAv8su4yOitqtBfaow9pWkhogo79W7ekBf+R7g71KJ+sr3AH+XYnmTlJmZFcSFYWZmBXFhvNkNaQcokb7yPcDfpRL1le8B/i5F8T4MMzMriNcwzMysIC6MdiR9RdJiSQslPSApl3am7pJ0naRnku/zC0kj0s7UXZI+KGmJpFZJve6IFkmzJC2XtELS1Wnn6S5Jt0jaIOnptLPsK0njJD0kaWny39Yn087UXZIGSnpc0qLku3y5bJ/lTVJvkDQsIl5Nnl8FTImIj6ccq1sknQH8d0Q0S/o6QET8fcqxukXSUUAr8D3gcxHRkHKkgknKAs8CpwOrgfnABRGxNNVg3SDpXcBrwA8i4pi08+wLSWOAMRHxhKShwALgL3rp34uAIRHxmqRq4HfAJyPi0VJ/ltcw2tlVFokhQK9t04h4ICKak8lHgbFp5tkXEbEsIpannaObZgArIuL5iGgCbgfOSTlTt0TEb4FNaecohYh4OSKeSJ7r5IuEAAAEJElEQVRvBZYBB6ebqnuizWvJZHXyKMtvlwujA0lflbQKuBD432nnKZGPAvelHaKfOhhY1W56Nb30h6mvkjQBeBvwWLpJuk9SVtJCYAMwLyLK8l36XWFI+rWkpzt5nAMQEV+MiHHAj4Er0027d119l2TMF4Fm2r5PxSrku5iVmqT9gJ8Bn+qwhaFXiYiWiJhO25aEGZLKssmwqhxvWski4rQCh/4YuBe4poxx9klX30XSpcDZwKlR4Turivh76W3WAOPaTY9N5lnKku39PwN+HBE/TztPKUTEZkkPAbOAkh+c0O/WMPZG0uR2k+cAz6SVZV9JmgX8HZCPiG1p5+nH5gOTJU2UVAN8CJiTcqZ+L9lRfDOwLCK+nXaefSGpdtdRkJIG0XaARVl+u3yUVDuSfgYcQdsROX8EPh4RvfJfg5JWAAOAjcmsR3vxEV/vB/4NqAU2Awsj4sx0UxVO0nuB7wBZ4JaI+GrKkbpF0k+AU2i7Kup64JqIuDnVUN0k6STgEeAp2v7/DvC/IuLe9FJ1j6RpwK20/feVAe6IiGvL8lkuDDMzK4Q3SZmZWUFcGGZmVhAXhpmZFcSFYWZmBXFhmJlZQVwYZmZWEBeGWTuSXut61F6Xv1PSocnz/SR9T9JKSQsk/UbSzGI+W9JnkktwL5b0oKTxyfxaSb/al6xmxXJhmJWIpKOBbEQ8n8y6ibaru06OiOOAj9B20lsxngTqI2IacCfwDYCIaARelnRiScKbFcCFYdYJtbkuuQDiU5LOT+ZnJH03uTnVPEn3SvrLZLELgbuScYcBM4EvRUQrQES8EBH3FJMjIh5qd2mXjpep/2XymWY9woVh1rkPANOBOuA04LrkpjsfACYAU4APA29vt8yJtN2IB+Bo2i5h0lLCTJfx5svUNwDvLOH7m+1Vv7tarVmBTgJ+kvzgr5f0MHB8Mv+/krWGdcmVQXcZAzSWI4yki4B64OR2szcAvfY2wtb7uDDMSmc7MDB5vgSok5Td17UMSacBXwROjogd7V4amHymWY/wJimzzj0CnJ/cyawWeBfwOPA/wLnJvozRtF29dZdlwCSAiFhJ2yajLyeX0kbSBEnvKyaEpLfRdi/zfERs6PDy4ZThngdme+I1DLPO/YK2/ROLaLs/8t9FxLrkEvinAktpu/XqE8CWZJl7aCuQXyfTlwPfAlZI2g68Anx+L585WNLqdtPfBt4L7Af8V9I7L0VEPnn93clnmvUIX97crEiS9ouI1yQdQNtax4lJmQwCHkqmS7mze085fgucExF/KvdnmYHXMMy64+7kDmc1wFciYh1ARGyXdA1wMPBSOQMkm8m+7bKwnuQ1DLMelKyVPNjJS6dGxMZO5ptVDBeGmZkVxEdJmZlZQVwYZmZWEBeGmZkVxIVhZmYFcWGYmVlB/j/OV097E7420gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "scores =  np.zeros((n_classes,n_Cs))\n",
    "mse_mean_L2= -np.mean(ls_L2.scores_[1],axis = 0)\n",
    "\n",
    "plt.plot(np.log10(Cs), mse_mean_L2.reshape(n_Cs,1)) \n",
    "\n",
    "plt.xlabel('log(C_L2)')\n",
    "plt.ylabel('neg-logloss_L2')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 239,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.60813127, 0.52170773, 0.49097572, 0.49131917, 0.49176285,\n",
       "       0.4918145 , 0.49181973])"
      ]
     },
     "execution_count": 239,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mse_mean_L2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 241,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.1 0.4909757237536471\n"
     ]
    }
   ],
   "source": [
    "best_C_L2 = np.argmin(mse_mean_L2)\n",
    "best_score_L2 = np.min(mse_mean_L2)\n",
    "print(Cs[best_C_L2], best_score_L2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
